package com.smartdo.crawler.processor;

import cn.hutool.core.thread.ThreadUtil;
import com.smartdo.crawler.utils.MongodbUtils;
import lombok.extern.slf4j.Slf4j;
import org.bson.Document;
import us.codecraft.webmagic.thread.CountableThreadPool;

import java.util.List;

/**
 * 解析数据多线程类
 *
 * @author zhangcheng
 */
@Slf4j
public class ProcessorJob {

    public static void main(String[] args) {
        new ProcessorJob().execute();
    }

    public ProcessorJob() {
        this(7);
    }

    public ProcessorJob(int theadNum) {
        threadPool = new CountableThreadPool(theadNum);
    }

    private CountableThreadPool threadPool;

    public void execute() {
        int skip = 0;
        int limit = 1000;
        while (true) {
            List<Document> dataList = MongodbUtils.getMongoDBInstance().find("test", "sumaitong", new Document(), skip, limit);
            if (dataList.size() == 0) {
                break;
            }
            for (int i = 0; i < dataList.size(); i++) {
                //线程池满时
                if (threadPool.getThreadAlive() == threadPool.getThreadNum()) {
                    ThreadUtil.sleep(500);
                    i--;
                    continue;
                }
                threadPool.execute(new ProcessorData(dataList.get(i)));
            }
            skip = skip + limit;
        }
    }
}